# -*- coding: utf-8 -*-
import json,time,scrapy,random

chrome = [
    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" ,
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
]
cate_info = {
    '数码':'512',
    '旅游':'510',
    '健康':'508',
    '时尚':'506',
    '美食':'505',
    '游戏':'504',
    '汽车':'503',
    '历史':'501',
    '房产':'513',
    '搞笑':'495',
    '教育':'497',
    '财经':'511',
    '科技':'496',
    '娱乐':'493',
    '体育':'507',
    '育儿':'499',
}
urls = {
    '娱乐':'http://newsapi.sina.cn/?resource=feed&mpName=%E5%A8%B1%E4%B9%90&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=4&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_ent&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_3281236c-353f-4815-b0a4-93411a0385f9&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=20&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1551963479&pullDirection=down&seId=7cab150e34&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213014%2C120.212195&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=54770db345&rand=472',
    '科技':'http://newsapi.sina.cn/?resource=feed&mpName=%E7%A7%91%E6%8A%80&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=6&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_tech&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_338a4468-9d9d-41c5-bab9-6409f06896ed&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=18&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1551960991&pullDirection=down&seId=7cab150e34&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213014%2C120.212195&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=d5c6eab3b9&rand=248',
    '教育':'http://newsapi.sina.cn/?resource=feed&mpName=%E6%95%99%E8%82%B2&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=4&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_edu&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_2ff2d9c6-e7fd-4ce5-a33a-5fa30bdbf8b2&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=18&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1551957334&pullDirection=down&seId=eefa23b84e&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=e8de0965f2&rand=136',
    '搞笑':'http://newsapi.sina.cn/?resource=feed&mpName=%E6%90%9E%E7%AC%91&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=5&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_funny&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_d0b1e7aa-61fc-4f4a-9a7c-f58230bb7aed&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=19&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1551954865&pullDirection=down&seId=eefa23b84e&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=1b19808abd&rand=568',
    '育儿':'http://newsapi.sina.cn/?resource=feed&mpName=%E8%82%B2%E5%84%BF&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=6&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_baby&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_a94b7018-f7eb-424d-8bda-991c661e9803&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=18&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1551955080&pullDirection=down&seId=eefa23b84e&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=bd439a2bbd&rand=506',
    '历史':'http://newsapi.sina.cn/?resource=feed&mpName=%E5%8E%86%E5%8F%B2&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=5&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_history&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_695936b6-942c-47be-97e2-07f21ef894b4&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=0&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=0&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=aba2dcdaf7&rand=755',
    '汽车':'http://newsapi.sina.cn/?resource=feed&mpName=%E6%B1%BD%E8%BD%A6&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=4&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_auto&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_6d605d3e-1864-4329-9384-6bde867a3550&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=20&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1546065366&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=4fa101842d&rand=528',
    '游戏':'http://newsapi.sina.cn/?resource=feed&mpName=%E6%B8%B8%E6%88%8F&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=6&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_game&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_579bf80f-6408-4137-bc4f-d2e95da6edb9&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=0&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=0&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=6ca31ac8c2&rand=608',
    '体育':'http://newsapi.sina.cn/?resource=feed&mpName=%E4%BD%93%E8%82%B2&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=6&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_sports&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_5c0d0545-b775-4d6a-b82a-43ecfbe1574c&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=22&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1551952665&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=2967db2d86&rand=734',
    '美食':'http://newsapi.sina.cn/?resource=feed&mpName=%E7%BE%8E%E9%A3%9F&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=5&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_food&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_adf9c436-8604-4d63-ba24-ee913f747806&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=0&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=0&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=5c97741c32&rand=521',
    '时尚':'http://newsapi.sina.cn/?resource=feed&mpName=%E6%97%B6%E5%B0%9A&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=4&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_fashion&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_343bcd36-8739-40f2-a26f-03e3edbff19a&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=16&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1551952661&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=46659f3660&rand=819',
    '健康':'http://newsapi.sina.cn/?resource=feed&mpName=%E5%81%A5%E5%BA%B7&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=5&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_health&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_8cd2bb7a-8fe0-4975-96ee-e7644e138a6f&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=18&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1551952753&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=2d90ab346a&rand=903',
    '财经':'http://newsapi.sina.cn/?resource=feed&mpName=%E8%B4%A2%E7%BB%8F&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=6&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_finance&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_d5a02b58-285a-431d-9cd7-43a9c92af797&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=20&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1546065366&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=481502b2c6&rand=615',
    '旅游':'http://newsapi.sina.cn/?resource=feed&mpName=%E6%97%85%E6%B8%B8&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=4&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_travel&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_bac129fd-e85f-45f6-b72a-ee6d774c0a91&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=19&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=1551952694&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=de1278f418&rand=562',
    '数码':'http://newsapi.sina.cn/?resource=feed&mpName=%E6%95%B0%E7%A0%81&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=4&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=news_digital&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_ffe4a2dd-9318-4b5c-b27f-922cf24e41b4&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=0&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=0&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=85eb2d54a7&rand=78',
    '房产':'http://newsapi.sina.cn/?resource=feed&mpName=%E6%88%BF%E4%BA%A7&lDid=1d1ede3f-6a20-479e-af55-63d3ef40b141&appVersion=7.11.1&oldChwm=14010_0001&upTimes=0&city=CHXX0044&loginType=1&prefetch=5&authToken=1a20c015016bc2d4d5327ffe7374cd50&channel=house_hangzhou&link=&authGuid=6470556192627389670&ua=Xiaomi-MI+6__sinanews__7.11.1__android__8.0.0&deviceId=e49d860d020b350a&connectionType=2&resolution=1080x1920&mac=02%3A00%3A00%3A00%3A00%3A00&weiboUid=5974644057&replacedFlag=0&osVersion=8.0.0&chwm=14010_0001&pullTimes=1&weiboSuid=08d65c9a33&andId=054e0de072fd4cf5&from=6000095012&sn=57b0728b&behavior=auto&aId=01AlM6hQgVxDm84BIkq1VY-F-FIhe2nV10MdNW9Q0w_05mm-s.&localSign=a_2f576a1f-8253-4c40-96c3-c4790dc5cbb9&deviceIdV1=e49d860d020b350a&todayReqTime=0&osSdk=26&abver=1551752314700110001&listCount=0&accessToken=2.00jqz1WGe3vYNC7aea426fd134pnHE&downTimes=0&lastTimestamp=0&pullDirection=down&seId=a08a484f44&imei=865873035293665&deviceModel=Xiaomi__Xiaomi__MI+6&location=30.213043%2C120.212184&authUid=6484667363936880619&loadingAdTimestamp=0&urlSign=6fed75bd1d&rand=37'
}

class SinaSpider(scrapy.Spider):
    name = 'sina'

    def start_requests(self):
        for cate,url in urls.items():
            yield scrapy.Request(url, callback=self.parse, headers={"User-Agent":random.choice(chrome)} ,meta={"cate":cate})

    def parse(self, response):
        cate = response.meta["cate"]
        res = json.loads(response.text).get('data').get('feed')
        for info in res:
            item = {}
            item['site_classify_id'] = int(cate_info.get(cate))
            item['title'] = info.get('title')
            if not item['title']:
                continue
            if info.get('category')=='video': #去掉视频
                continue
            if info.get('category')=='hdpic': #去掉图集
                continue
            if item['site_classify_id']==501:
                item['publish_time'] = time.strftime("%Y-%m-%d %X", time.localtime(info.get('articlePubDate')))
            else:
                item['publish_time'] = time.strftime("%Y-%m-%d %X", time.localtime(info.get('pubDate')))
            source = info.get('source').replace('新浪财经-','')
            item['author'] = source if source else "新浪新闻"
            item['comment_count'] = int(info.get('comment')) if info.get('comment') else 0
            if item['comment_count'] != 0:
                item['browse_count']=item['comment_count']*random.randint(100,200)
            else:
                item['browse_count']=random.randint(100,3000)
            item['site_id'] = 114 #114表示新浪
            item['url'] = info.get('link')
            yield item

